import shutil
import time
import traceback
from multiprocessing import Manager, Process

import facade
from bs4 import BeautifulSoup
from xjlibrary.msqlite3.sqlite3_client import sqliteEscape
from xjlibrary.our_file_dir import BaseDir

curPath = BaseDir.get_file_dir_absolute(__file__)
TopPath = BaseDir.get_upper_dir(curPath, -2)
sPath = BaseDir.get_new_path(TopPath, "download", "jstor", "download", "issue")  # volume
sPath2 = BaseDir.get_new_path(TopPath, "download", "jstor", "download", "issue2")
BaseDir.create_dir(sPath2)
configfile = BaseDir.get_new_path(curPath, "db.ini")


def setTask(queue):
    """
    多进程设置队列任务函数
    :param queue:
    :return:
    """
    print("进入setTask")
    while True:
        for filepath in BaseDir.get_dir_all_files(sPath):
            print("put file:"+filepath)
            queue.put(filepath)
        print("time.sleep(30)")
        time.sleep(30)


def parahtml(queue, queueresult):
    while True:
        filePath = queue.get()
        print("get file:" + filePath)
        try:
            filename = BaseDir.get_filename_not_extsep(filePath)
            iurl = filename.replace("_", "/")
            issueurl = "/stable" + iurl
            soup = BeautifulSoup(open(filePath, 'r', encoding='utf-8'), "lxml")
            # print(soup)
            ol_tag = soup.find("ol", class_="no-bullet")
            if not ol_tag:
                return
            a_all_tag = ol_tag.find_all("a", class_="tt-track small-heading inline", attrs={"data-qa": "content title"})
            num = len(a_all_tag)
            for a_tag in a_all_tag:
                href = a_tag["href"]
                title = "".join(a_tag.stripped_strings)
                sql = "INSERT IGNORE INTO `article` (`url`,`title`,`issueurl`,`num`) VALUES('{url}','{title}','{issueurl}','{num}')"
                sql = sql.format(url=href, title=sqliteEscape(title), issueurl=sqliteEscape(issueurl), num=num)
                queueresult.put(sql)
            shutil.move(filePath, sPath2)
        except:
            traceback.print_exc()


def InsertSql(queueresult):
    mysqlutils = facade.MysqlUtiles(configfile, "db")
    Listsql = []
    while True:
        if queueresult.empty():
            print(len(Listsql))
            mysqlutils.ExeSqlListToDB(Listsql)
            Listsql.clear()
        Listsql.append(queueresult.get())
        if len(Listsql) > 100:
            print(len(Listsql))
            mysqlutils.ExeSqlListToDB(Listsql)
            Listsql.clear()


if __name__ == "__main__":
    Process_list = []
    with Manager() as manager:
        queue = manager.Queue(300)
        queueresult = manager.Queue(300)
        Process_list.append(Process(target=setTask, args=(queue,)))
        Process_list.append(Process(target=InsertSql, args=(queueresult,)))
        for _ in range(0, 3):
            Process_list.append(Process(target=parahtml, args=(queue, queueresult)))
        for process in Process_list:
            process.start()
        for process in Process_list:
            process.join()
